<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=9"/>
<meta name="generator" content="Doxygen 1.8.6"/>
<title>JSON Voorhees: jsonv::tokenizer Class Reference</title>
<link href="tabs.css" rel="stylesheet" type="text/css"/>
<script type="text/javascript" src="jquery.js"></script>
<script type="text/javascript" src="dynsections.js"></script>
<link href="navtree.css" rel="stylesheet" type="text/css"/>
<script type="text/javascript" src="resize.js"></script>
<script type="text/javascript" src="navtree.js"></script>
<script type="text/javascript">
  $(document).ready(initResizable);
  $(window).load(resizeHeight);
</script>
<link href="search/search.css" rel="stylesheet" type="text/css"/>
<script type="text/javascript" src="search/search.js"></script>
<script type="text/javascript">
  $(document).ready(function() { searchBox.OnSelectItem(0); });
</script>
<script type="text/x-mathjax-config">
  MathJax.Hub.Config({
    extensions: ["tex2jax.js"],
    jax: ["input/TeX","output/HTML-CSS"],
});
</script><script src="http://www.mathjax.org/mathjax/MathJax.js"></script>
<link href="doxygen.css" rel="stylesheet" type="text/css" />
</head>
<body>
<div id="top"><!-- do not remove this div, it is closed by doxygen! -->
<div id="titlearea">
<table cellspacing="0" cellpadding="0">
 <tbody>
 <tr style="height: 56px;">
  <td style="padding-left: 0.5em;">
   <div id="projectname">JSON Voorhees
   </div>
   <div id="projectbrief">Killer JSON for C++</div>
  </td>
 </tr>
 </tbody>
</table>
</div>
<!-- end header part -->
<!-- Generated by Doxygen 1.8.6 -->
<script type="text/javascript">
var searchBox = new SearchBox("searchBox", "search",false,'Search');
</script>
  <div id="navrow1" class="tabs">
    <ul class="tablist">
      <li><a href="index.html"><span>Main&#160;Page</span></a></li>
      <li><a href="pages.html"><span>Related&#160;Pages</span></a></li>
      <li><a href="modules.html"><span>Modules</span></a></li>
      <li class="current"><a href="annotated.html"><span>Classes</span></a></li>
      <li><a href="files.html"><span>Files</span></a></li>
      <li><a href="examples.html"><span>Examples</span></a></li>
      <li>
        <div id="MSearchBox" class="MSearchBoxInactive">
        <span class="left">
          <img id="MSearchSelect" src="search/mag_sel.png"
               onmouseover="return searchBox.OnSearchSelectShow()"
               onmouseout="return searchBox.OnSearchSelectHide()"
               alt=""/>
          <input type="text" id="MSearchField" value="Search" accesskey="S"
               onfocus="searchBox.OnSearchFieldFocus(true)" 
               onblur="searchBox.OnSearchFieldFocus(false)" 
               onkeyup="searchBox.OnSearchFieldChange(event)"/>
          </span><span class="right">
            <a id="MSearchClose" href="javascript:searchBox.CloseResultsWindow()"><img id="MSearchCloseImg" border="0" src="search/close.png" alt=""/></a>
          </span>
        </div>
      </li>
    </ul>
  </div>
  <div id="navrow2" class="tabs2">
    <ul class="tablist">
      <li><a href="annotated.html"><span>Class&#160;List</span></a></li>
      <li><a href="classes.html"><span>Class&#160;Index</span></a></li>
      <li><a href="hierarchy.html"><span>Class&#160;Hierarchy</span></a></li>
      <li><a href="functions.html"><span>Class&#160;Members</span></a></li>
    </ul>
  </div>
</div><!-- top -->
<div id="side-nav" class="ui-resizable side-nav-resizable">
  <div id="nav-tree">
    <div id="nav-tree-contents">
      <div id="nav-sync" class="sync"></div>
    </div>
  </div>
  <div id="splitbar" style="-moz-user-select:none;" 
       class="ui-resizable-handle">
  </div>
</div>
<script type="text/javascript">
$(document).ready(function(){initNavTree('classjsonv_1_1tokenizer.html','');});
</script>
<div id="doc-content">
<!-- window showing the filter options -->
<div id="MSearchSelectWindow"
     onmouseover="return searchBox.OnSearchSelectShow()"
     onmouseout="return searchBox.OnSearchSelectHide()"
     onkeydown="return searchBox.OnSearchSelectKey(event)">
<a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(0)"><span class="SelectionMark">&#160;</span>All</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(1)"><span class="SelectionMark">&#160;</span>Classes</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(2)"><span class="SelectionMark">&#160;</span>Files</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(3)"><span class="SelectionMark">&#160;</span>Functions</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(4)"><span class="SelectionMark">&#160;</span>Variables</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(5)"><span class="SelectionMark">&#160;</span>Typedefs</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(6)"><span class="SelectionMark">&#160;</span>Enumerations</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(7)"><span class="SelectionMark">&#160;</span>Enumerator</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(8)"><span class="SelectionMark">&#160;</span>Friends</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(9)"><span class="SelectionMark">&#160;</span>Macros</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(10)"><span class="SelectionMark">&#160;</span>Groups</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(11)"><span class="SelectionMark">&#160;</span>Pages</a></div>

<!-- iframe showing the search results (closed by default) -->
<div id="MSearchResultsWindow">
<iframe src="javascript:void(0)" frameborder="0" 
        name="MSearchResults" id="MSearchResults">
</iframe>
</div>

<div class="header">
  <div class="summary">
<a href="#nested-classes">Classes</a> &#124;
<a href="#pub-types">Public Types</a> &#124;
<a href="#pub-methods">Public Member Functions</a> &#124;
<a href="#pub-static-methods">Static Public Member Functions</a> &#124;
<a href="classjsonv_1_1tokenizer-members.html">List of all members</a>  </div>
  <div class="headertitle">
<div class="title">jsonv::tokenizer Class Reference</div>  </div>
</div><!--header-->
<div class="contents">

<p>Splits input into tokens, allowing traversal of JSON without verification.  
 <a href="classjsonv_1_1tokenizer.html#details">More...</a></p>

<p><code>#include &lt;<a class="el" href="tokenizer_8hpp_source.html">jsonv/tokenizer.hpp</a>&gt;</code></p>
<table class="memberdecls">
<tr class="heading"><td colspan="2"><h2 class="groupheader"><a name="nested-classes"></a>
Classes</h2></td></tr>
<tr class="memitem:"><td class="memItemLeft" align="right" valign="top">struct &#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="structjsonv_1_1tokenizer_1_1token.html">token</a></td></tr>
<tr class="memdesc:"><td class="mdescLeft">&#160;</td><td class="mdescRight">A representation of what this tokenizer has.  <a href="structjsonv_1_1tokenizer_1_1token.html#details">More...</a><br/></td></tr>
<tr class="separator:"><td class="memSeparator" colspan="2">&#160;</td></tr>
</table><table class="memberdecls">
<tr class="heading"><td colspan="2"><h2 class="groupheader"><a name="pub-types"></a>
Public Types</h2></td></tr>
<tr class="memitem:a14376a352de44485a5c87138a7f1e206"><td class="memItemLeft" align="right" valign="top"><a class="anchor" id="a14376a352de44485a5c87138a7f1e206"></a>
using&#160;</td><td class="memItemRight" valign="bottom"><b>size_type</b> = std::vector&lt; char &gt;::size_type</td></tr>
<tr class="separator:a14376a352de44485a5c87138a7f1e206"><td class="memSeparator" colspan="2">&#160;</td></tr>
</table><table class="memberdecls">
<tr class="heading"><td colspan="2"><h2 class="groupheader"><a name="pub-methods"></a>
Public Member Functions</h2></td></tr>
<tr class="memitem:aef9d89558ab9ca24c9e833eb2f6da69a"><td class="memItemLeft" align="right" valign="top"><a class="anchor" id="aef9d89558ab9ca24c9e833eb2f6da69a"></a>
&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="classjsonv_1_1tokenizer.html#aef9d89558ab9ca24c9e833eb2f6da69a">tokenizer</a> (string_view <a class="el" href="classjsonv_1_1tokenizer.html#ab10fa4c3c04c2029f6a40b7113a9282b">input</a>)</td></tr>
<tr class="memdesc:aef9d89558ab9ca24c9e833eb2f6da69a"><td class="mdescLeft">&#160;</td><td class="mdescRight">Construct a tokenizer to read the given non-owned <em>input</em>. <br/></td></tr>
<tr class="separator:aef9d89558ab9ca24c9e833eb2f6da69a"><td class="memSeparator" colspan="2">&#160;</td></tr>
<tr class="memitem:ab6e085d39aa9aee3ba2e87cefdbd24fb"><td class="memItemLeft" align="right" valign="top"><a class="anchor" id="ab6e085d39aa9aee3ba2e87cefdbd24fb"></a>
&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="classjsonv_1_1tokenizer.html#ab6e085d39aa9aee3ba2e87cefdbd24fb">tokenizer</a> (std::istream &amp;<a class="el" href="classjsonv_1_1tokenizer.html#ab10fa4c3c04c2029f6a40b7113a9282b">input</a>)</td></tr>
<tr class="memdesc:ab6e085d39aa9aee3ba2e87cefdbd24fb"><td class="mdescLeft">&#160;</td><td class="mdescRight">Construct a tokenizer from the provided <em>input</em>. <br/></td></tr>
<tr class="separator:ab6e085d39aa9aee3ba2e87cefdbd24fb"><td class="memSeparator" colspan="2">&#160;</td></tr>
<tr class="memitem:ab10fa4c3c04c2029f6a40b7113a9282b"><td class="memItemLeft" align="right" valign="top"><a class="anchor" id="ab10fa4c3c04c2029f6a40b7113a9282b"></a>
const string_view &amp;&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="classjsonv_1_1tokenizer.html#ab10fa4c3c04c2029f6a40b7113a9282b">input</a> () const </td></tr>
<tr class="memdesc:ab10fa4c3c04c2029f6a40b7113a9282b"><td class="mdescLeft">&#160;</td><td class="mdescRight">Get the input this instance is reading from. <br/></td></tr>
<tr class="separator:ab10fa4c3c04c2029f6a40b7113a9282b"><td class="memSeparator" colspan="2">&#160;</td></tr>
<tr class="memitem:a6cb4f97577999ebc4dd898acdf6f3454"><td class="memItemLeft" align="right" valign="top">bool&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="classjsonv_1_1tokenizer.html#a6cb4f97577999ebc4dd898acdf6f3454">next</a> ()</td></tr>
<tr class="memdesc:a6cb4f97577999ebc4dd898acdf6f3454"><td class="mdescLeft">&#160;</td><td class="mdescRight">Attempt to go to the next token in the input stream.  <a href="#a6cb4f97577999ebc4dd898acdf6f3454">More...</a><br/></td></tr>
<tr class="separator:a6cb4f97577999ebc4dd898acdf6f3454"><td class="memSeparator" colspan="2">&#160;</td></tr>
<tr class="memitem:a738c2b782c3fa1b1a1c9293b9d1c6cb4"><td class="memItemLeft" align="right" valign="top">const <a class="el" href="structjsonv_1_1tokenizer_1_1token.html">token</a> &amp;&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="classjsonv_1_1tokenizer.html#a738c2b782c3fa1b1a1c9293b9d1c6cb4">current</a> () const </td></tr>
<tr class="memdesc:a738c2b782c3fa1b1a1c9293b9d1c6cb4"><td class="mdescLeft">&#160;</td><td class="mdescRight">Get the current token and its associated <code>token_kind</code>.  <a href="#a738c2b782c3fa1b1a1c9293b9d1c6cb4">More...</a><br/></td></tr>
<tr class="separator:a738c2b782c3fa1b1a1c9293b9d1c6cb4"><td class="memSeparator" colspan="2">&#160;</td></tr>
<tr class="memitem:a23b1a2cf8aa90aa52e4f0a104577e15a"><td class="memItemLeft" align="right" valign="top">void&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="classjsonv_1_1tokenizer.html#a23b1a2cf8aa90aa52e4f0a104577e15a">buffer_reserve</a> (size_type sz)</td></tr>
<tr class="separator:a23b1a2cf8aa90aa52e4f0a104577e15a"><td class="memSeparator" colspan="2">&#160;</td></tr>
</table><table class="memberdecls">
<tr class="heading"><td colspan="2"><h2 class="groupheader"><a name="pub-static-methods"></a>
Static Public Member Functions</h2></td></tr>
<tr class="memitem:aeb60c50524ebcb6dad94eb9ad639a984"><td class="memItemLeft" align="right" valign="top">static size_type&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="classjsonv_1_1tokenizer.html#aeb60c50524ebcb6dad94eb9ad639a984">min_buffer_size</a> ()</td></tr>
<tr class="separator:aeb60c50524ebcb6dad94eb9ad639a984"><td class="memSeparator" colspan="2">&#160;</td></tr>
<tr class="memitem:adcbc1833dec761c8e01b0390aa816e95"><td class="memItemLeft" align="right" valign="top">static void&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="classjsonv_1_1tokenizer.html#adcbc1833dec761c8e01b0390aa816e95">set_min_buffer_size</a> (size_type sz)</td></tr>
<tr class="separator:adcbc1833dec761c8e01b0390aa816e95"><td class="memSeparator" colspan="2">&#160;</td></tr>
</table>
<a name="details" id="details"></a><h2 class="groupheader">Detailed Description</h2>
<div class="textblock"><p>Splits input into tokens, allowing traversal of JSON without verification. </p>
<p>This is the basis for JSON parsers.</p>
<p>An important thing to remember is a <code>tokenizer</code> does not perform any real validation of any kind beyond emitting a <code>token_kind::unknown</code> when it encounters complete garbage. What does this mean? Given the input string:</p>
<div class="fragment"><div class="line">[{]{{{{{{}}{{]]]]][][]]][[[[]]]<span class="stringliteral">&quot;fdsadf&quot;</span>]]]}}}}}}}}]]]</div>
</div><!-- fragment --><p>A <code>tokenizer</code> will emit <code>token_kind::array_begin</code>, <code>token_kind::object_begin</code>, <code>token_kind::array_end</code> and so on, even though it is illegal JSON. It is up to a higher-level construct to detect such failures. </p>
<dl><dt><b>Examples: </b></dt><dd><a class="el" href="parse_07tokenizer_6_00_01const_01parse_options_6_08-example.html#_a0">parse(tokenizer&amp;, const parse_options&amp;)</a>.</dd>
</dl>
<p>Definition at line <a class="el" href="tokenizer_8hpp_source.html#l00099">99</a> of file <a class="el" href="tokenizer_8hpp_source.html">tokenizer.hpp</a>.</p>
</div><h2 class="groupheader">Member Function Documentation</h2>
<a class="anchor" id="a23b1a2cf8aa90aa52e4f0a104577e15a"></a>
<div class="memitem">
<div class="memproto">
      <table class="memname">
        <tr>
          <td class="memname">void jsonv::tokenizer::buffer_reserve </td>
          <td>(</td>
          <td class="paramtype">size_type&#160;</td>
          <td class="paramname"><em>sz</em></td><td>)</td>
          <td></td>
        </tr>
      </table>
</div><div class="memdoc">
<dl class="deprecated"><dt><b><a class="el" href="deprecated.html#_deprecated000003">Deprecated:</a></b></dt><dd>Calling this function has no effect and will be removed in 2.0. </dd></dl>

</div>
</div>
<a class="anchor" id="a738c2b782c3fa1b1a1c9293b9d1c6cb4"></a>
<div class="memitem">
<div class="memproto">
      <table class="memname">
        <tr>
          <td class="memname">const <a class="el" href="structjsonv_1_1tokenizer_1_1token.html">token</a>&amp; jsonv::tokenizer::current </td>
          <td>(</td>
          <td class="paramname"></td><td>)</td>
          <td> const</td>
        </tr>
      </table>
</div><div class="memdoc">

<p>Get the current token and its associated <code>token_kind</code>. </p>
<dl class="section return"><dt>Returns</dt><dd>The current token. </dd></dl>
<dl class="exception"><dt>Exceptions</dt><dd>
  <table class="exception">
    <tr><td class="paramname">std::logic_error</td><td>if <code>next</code> has not been called or if it returned <code>false</code>. </td></tr>
  </table>
  </dd>
</dl>

</div>
</div>
<a class="anchor" id="aeb60c50524ebcb6dad94eb9ad639a984"></a>
<div class="memitem">
<div class="memproto">
<table class="mlabels">
  <tr>
  <td class="mlabels-left">
      <table class="memname">
        <tr>
          <td class="memname">static size_type jsonv::tokenizer::min_buffer_size </td>
          <td>(</td>
          <td class="paramname"></td><td>)</td>
          <td></td>
        </tr>
      </table>
  </td>
  <td class="mlabels-right">
<span class="mlabels"><span class="mlabel">static</span></span>  </td>
  </tr>
</table>
</div><div class="memdoc">
<dl class="deprecated"><dt><b><a class="el" href="deprecated.html#_deprecated000001">Deprecated:</a></b></dt><dd>See <a class="el" href="classjsonv_1_1tokenizer.html#a23b1a2cf8aa90aa52e4f0a104577e15a">buffer_reserve</a>. </dd></dl>

</div>
</div>
<a class="anchor" id="a6cb4f97577999ebc4dd898acdf6f3454"></a>
<div class="memitem">
<div class="memproto">
      <table class="memname">
        <tr>
          <td class="memname">bool jsonv::tokenizer::next </td>
          <td>(</td>
          <td class="paramname"></td><td>)</td>
          <td></td>
        </tr>
      </table>
</div><div class="memdoc">

<p>Attempt to go to the next token in the input stream. </p>
<p>The contents of <code>current</code> will be cleared.</p>
<dl class="section return"><dt>Returns</dt><dd><code>true</code> if another token was obtained; <code>false</code> if we reached EOF or an I/O failure. Check <code>input</code> to see which. </dd></dl>

</div>
</div>
<a class="anchor" id="adcbc1833dec761c8e01b0390aa816e95"></a>
<div class="memitem">
<div class="memproto">
<table class="mlabels">
  <tr>
  <td class="mlabels-left">
      <table class="memname">
        <tr>
          <td class="memname">static void jsonv::tokenizer::set_min_buffer_size </td>
          <td>(</td>
          <td class="paramtype">size_type&#160;</td>
          <td class="paramname"><em>sz</em></td><td>)</td>
          <td></td>
        </tr>
      </table>
  </td>
  <td class="mlabels-right">
<span class="mlabels"><span class="mlabel">static</span></span>  </td>
  </tr>
</table>
</div><div class="memdoc">
<dl class="deprecated"><dt><b><a class="el" href="deprecated.html#_deprecated000002">Deprecated:</a></b></dt><dd>See <a class="el" href="classjsonv_1_1tokenizer.html#a23b1a2cf8aa90aa52e4f0a104577e15a">buffer_reserve</a>. </dd></dl>

</div>
</div>
<hr/>The documentation for this class was generated from the following file:<ul>
<li>jsonv/<a class="el" href="tokenizer_8hpp_source.html">tokenizer.hpp</a></li>
</ul>
</div><!-- contents -->
</div><!-- doc-content -->
<!-- start footer part -->
<div id="nav-path" class="navpath"><!-- id is needed for treeview function! -->
  <ul>
    <li class="navelem"><b>jsonv</b></li><li class="navelem"><a class="el" href="classjsonv_1_1tokenizer.html">tokenizer</a></li>
    <li class="footer">Generated by
    <a href="http://www.doxygen.org/index.html">
    <img class="footer" src="doxygen.png" alt="doxygen"/></a> 1.8.6 </li>
  </ul>
</div>
</body>
</html>
